library(tidyverse)
library(plotly)
RESEARCH_HOME = "/home/and/Documents/PhD/Research/"
FILES_PATH = "Scripts/R/Benchmarks/MultiAndSingleNode/R12/dblab/"
MONITOR_FILE = "monitorDemo.log"
NOHUP_FILE = "nohupDemo.out"
SEPARATOR_ID = "-"
lines = readLines(paste0(RESEARCH_HOME, FILES_PATH, NOHUP_FILE))
lines = lines[grepl("\\|[1-6]\\.", lines)]
nohup = as_tibble(lines) %>%
separate(value, into=c("Timestamp", "ID", "Time", "Stage", "Duration", "Load", "TS"), sep="\\|") %>%
separate(ID, into=c(NA, NA, "ID"), sep = SEPARATOR_ID) %>%
mutate(ID = as.numeric(ID), Time = as.numeric(Time), Duration = as.numeric(Duration))
nohupStages = nohup %>% mutate(Stage = paste0(TS,".",str_trim(Stage))) %>%
mutate(Start = Time - Duration, End = Time) %>%
select(ID, Stage, Start, End, Duration)
nohupTimeintervals = nohup %>% select(ID, TS, Time, Duration) %>%
mutate(Start = Time - Duration, End = Time) %>%
group_by(ID, TS) %>% summarise(Start = min(Start), End = max(End))
checkF <- function(ID, Stage, TS, Time, Start, End){
check = Start < Time && Time < End
x <- as_tibble(check)
}
getStageStart <- function(d){
join = d %>% left_join(nohupTimeintervals, by = "ID")
checks = j %>% select(ID, Stage, TS, Time, Start, End) %>% pmap_dfr(checkF)
starts = cbind(join, checks) %>% filter(value) %>%
mutate(Stage = paste0(TS,".",Stage)) %>%
select(Stage, Nodes, Time, y) %>%
group_by(Stage, Nodes) %>%
summarise(Start=min(Time), y=min(y)) %>% arrange(Start)
return(starts)
}
lines = readLines(paste0(RESEARCH_HOME, FILES_PATH, MONITOR_FILE))
lines = lines[grepl("\\|SCALE\\|", lines)]
monitor = as_tibble(lines) %>%
separate(value, into=c("Timestamp", "Scale", "Time", "ID", "Nodes", "Stage", "RDDs", "Task", "Dura", "Load"), sep="\\|") %>%
separate(ID, into=c(NA, NA, "ID"), sep = SEPARATOR_ID) %>%
select(ID, Time, Nodes, Stage, RDDs, Task, Load) %>%
mutate(ID=as.numeric(ID), Time=as.numeric(Time), RDDs=as.numeric(RDDs), Tasks=as.numeric(Task), Load=as.numeric(Load)) %>%
group_by(ID, Time, Nodes, Stage) %>% summarise(RDDs=mean(RDDs), Tasks=mean(Tasks), Load=mean(Load))
head(monitor)
apps = monitor %>% ungroup() %>% select(ID, Nodes) %>% distinct()
head(apps, n=Inf)
d = monitor %>% filter(ID %in% c(0,1,2)) %>% ungroup %>%
mutate(y = Load) %>%
select(ID, Time, y, Nodes, Stage) %>%
arrange(Time)
stageStarts =getStageStart(d)
p = ggplot(data = d, aes(x = Time, y = y, group = 1, color = Nodes, linetype = Nodes)) +
geom_line() +
geom_point(data = stageStarts, aes(x=Start, y=y, group=1, color=Nodes, text = paste(Stage,"<br>Start:",Start,"<br>",y)))
Ignoring unknown aesthetics: text
ggplotly(p, tooltip = c("text"))
LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KHBsb3RseSkKClJFU0VBUkNIX0hPTUUgPSAiL2hvbWUvYW5kL0RvY3VtZW50cy9QaEQvUmVzZWFyY2gvIgpGSUxFU19QQVRIID0gIlNjcmlwdHMvUi9CZW5jaG1hcmtzL011bHRpQW5kU2luZ2xlTm9kZS9SMTIvZGJsYWIvIgpNT05JVE9SX0ZJTEUgPSAibW9uaXRvckRlbW8ubG9nIgpOT0hVUF9GSUxFID0gIm5vaHVwRGVtby5vdXQiClNFUEFSQVRPUl9JRCA9ICItIgoKbGluZXMgPSByZWFkTGluZXMocGFzdGUwKFJFU0VBUkNIX0hPTUUsIEZJTEVTX1BBVEgsIE5PSFVQX0ZJTEUpKQpsaW5lcyA9IGxpbmVzW2dyZXBsKCJcXHxbMS02XVxcLiIsIGxpbmVzKV0Kbm9odXAgPSBhc190aWJibGUobGluZXMpICU+JQogIHNlcGFyYXRlKHZhbHVlLCBpbnRvPWMoIlRpbWVzdGFtcCIsICJJRCIsICJUaW1lIiwgIlN0YWdlIiwgIkR1cmF0aW9uIiwgIkxvYWQiLCAiVFMiKSwgc2VwPSJcXHwiKSAlPiUKICBzZXBhcmF0ZShJRCwgaW50bz1jKE5BLCBOQSwgIklEIiksIHNlcCA9IFNFUEFSQVRPUl9JRCkgJT4lCiAgbXV0YXRlKElEID0gYXMubnVtZXJpYyhJRCksIFRpbWUgPSBhcy5udW1lcmljKFRpbWUpLCBEdXJhdGlvbiA9IGFzLm51bWVyaWMoRHVyYXRpb24pKSAKICAKbm9odXBTdGFnZXMgPSBub2h1cCAlPiUgbXV0YXRlKFN0YWdlID0gcGFzdGUwKFRTLCIuIixzdHJfdHJpbShTdGFnZSkpKSAlPiUKICBtdXRhdGUoU3RhcnQgPSBUaW1lIC0gRHVyYXRpb24sIEVuZCA9IFRpbWUpICU+JQogIHNlbGVjdChJRCwgU3RhZ2UsIFN0YXJ0LCBFbmQsIER1cmF0aW9uKSAKCm5vaHVwVGltZWludGVydmFscyA9IG5vaHVwICU+JSBzZWxlY3QoSUQsIFRTLCBUaW1lLCBEdXJhdGlvbikgJT4lCiAgbXV0YXRlKFN0YXJ0ID0gVGltZSAtIER1cmF0aW9uLCBFbmQgPSBUaW1lKSAlPiUKICBncm91cF9ieShJRCwgVFMpICU+JSBzdW1tYXJpc2UoU3RhcnQgPSBtaW4oU3RhcnQpLCBFbmQgPSBtYXgoRW5kKSkKCmNoZWNrRiA8LSBmdW5jdGlvbihJRCwgU3RhZ2UsIFRTLCBUaW1lLCBTdGFydCwgRW5kKXsgCiAgY2hlY2sgPSBTdGFydCA8IFRpbWUgJiYgVGltZSA8IEVuZAogIHggPC0gYXNfdGliYmxlKGNoZWNrKSAKfQoKZ2V0U3RhZ2VTdGFydCA8LSBmdW5jdGlvbihkKXsKICBqb2luID0gZCAlPiUgbGVmdF9qb2luKG5vaHVwVGltZWludGVydmFscywgYnkgPSAiSUQiKQogIGNoZWNrcyA9IGpvaW4gJT4lIHNlbGVjdChJRCwgU3RhZ2UsIFRTLCBUaW1lLCBTdGFydCwgRW5kKSAlPiUgcG1hcF9kZnIoY2hlY2tGKQogIHN0YXJ0cyA9IGNiaW5kKGpvaW4sIGNoZWNrcykgJT4lIGZpbHRlcih2YWx1ZSkgJT4lIAogICAgbXV0YXRlKFN0YWdlID0gcGFzdGUwKFRTLCIuIixTdGFnZSkpICU+JQogICAgc2VsZWN0KFN0YWdlLCBOb2RlcywgVGltZSwgeSkgJT4lIAogICAgZ3JvdXBfYnkoU3RhZ2UsIE5vZGVzKSAlPiUgCiAgICBzdW1tYXJpc2UoU3RhcnQ9bWluKFRpbWUpLCB5PW1pbih5KSkgJT4lIGFycmFuZ2UoU3RhcnQpCiAgcmV0dXJuKHN0YXJ0cykKfQoKCmBgYAoKYGBge3J9CmxpbmVzID0gcmVhZExpbmVzKHBhc3RlMChSRVNFQVJDSF9IT01FLCBGSUxFU19QQVRILCBNT05JVE9SX0ZJTEUpKQpsaW5lcyA9IGxpbmVzW2dyZXBsKCJcXHxTQ0FMRVxcfCIsIGxpbmVzKV0KbW9uaXRvciA9IGFzX3RpYmJsZShsaW5lcykgJT4lCiAgc2VwYXJhdGUodmFsdWUsIGludG89YygiVGltZXN0YW1wIiwgIlNjYWxlIiwgIlRpbWUiLCAiSUQiLCAiTm9kZXMiLCAiU3RhZ2UiLCAiUkREcyIsICJUYXNrIiwgIkR1cmEiLCAiTG9hZCIpLCBzZXA9IlxcfCIpICU+JQogIHNlcGFyYXRlKElELCBpbnRvPWMoTkEsIE5BLCAiSUQiKSwgc2VwID0gU0VQQVJBVE9SX0lEKSAlPiUKICBzZWxlY3QoSUQsIFRpbWUsIE5vZGVzLCBTdGFnZSwgUkREcywgVGFzaywgTG9hZCkgJT4lCiAgbXV0YXRlKElEPWFzLm51bWVyaWMoSUQpLCBUaW1lPWFzLm51bWVyaWMoVGltZSksIFJERHM9YXMubnVtZXJpYyhSRERzKSwgVGFza3M9YXMubnVtZXJpYyhUYXNrKSwgTG9hZD1hcy5udW1lcmljKExvYWQpKSAlPiUKICBncm91cF9ieShJRCwgVGltZSwgTm9kZXMsIFN0YWdlKSAlPiUgc3VtbWFyaXNlKFJERHM9bWVhbihSRERzKSwgVGFza3M9bWVhbihUYXNrcyksIExvYWQ9bWVhbihMb2FkKSkKaGVhZChtb25pdG9yKQpgYGAKCmBgYHtyfQphcHBzID0gbW9uaXRvciAlPiUgdW5ncm91cCgpICU+JSBzZWxlY3QoSUQsIE5vZGVzKSAlPiUgZGlzdGluY3QoKQpoZWFkKGFwcHMsIG49SW5mKQpgYGAKCgpgYGB7cn0KZCA9IG1vbml0b3IgJT4lIGZpbHRlcihJRCAlaW4lIGMoMCwxLDIpKSAlPiUgdW5ncm91cCAlPiUgCiAgbXV0YXRlKHkgPSBMb2FkKSAlPiUKICBzZWxlY3QoSUQsIFRpbWUsIHksIE5vZGVzLCBTdGFnZSkgJT4lIAogIGFycmFuZ2UoVGltZSkgCgpzdGFnZVN0YXJ0cyA9Z2V0U3RhZ2VTdGFydChkKQoKCmBgYAoKYGBge3J9CnAgPSBnZ3Bsb3QoZGF0YSA9IGQsIGFlcyh4ID0gVGltZSwgeSA9IHksIGdyb3VwID0gMSwgY29sb3IgPSBOb2RlcywgbGluZXR5cGUgPSBOb2RlcykpICsKICBnZW9tX2xpbmUoKSArCiAgZ2VvbV9wb2ludChkYXRhID0gc3RhZ2VTdGFydHMsIGFlcyh4PVN0YXJ0LCB5PXksIGdyb3VwPTEsIGNvbG9yPU5vZGVzLCB0ZXh0ID0gcGFzdGUoU3RhZ2UsIjxicj5TdGFydDoiLFN0YXJ0LCI8YnI+Iix5KSkpCmBgYAoKYGBge3J9CmdncGxvdGx5KHAsIHRvb2x0aXAgPSBjKCJ0ZXh0IikpCmBgYAoKCg==